package com.g2room.driving.grap.article.qg;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrabYszdDataByProxy {
	static String imgPath = "ys";
	public String baseUrl = "http://www.ys137.com";
	private String soruce = "养生之道网";
	private int count = 0;
	static List<String> titles = GenericArti.getTitles();
	
	public static void main(String[] args) throws Exception{
		GrabYszdDataByProxy gkd = new GrabYszdDataByProxy();
//		gkd.getNav();
		
		gkd.grapImages();
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
		
		
		
//		http://www.ys137.com/xiuxian/qgks/list_240_63.html
		pages.put("http://www.ys137.com/xiuxian/qgks/list_240_183.html", "情感口述");
//		pages.put("http://www.ys137.com/xiuxian/pxgx/list_239_163.html", "婆媳关系");
//		pages.put("http://www.ys137.com/xiuxian/wqnw/list_241_132.html", "围城内外-婚姻生活");
 
        for(String key : pages.keySet()) {
	        String pageTotal = key.substring(key.lastIndexOf("_") + 1, key.lastIndexOf("."));
	        for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
	        	if(i<170 || i>190) continue;
	        	String url = key.replaceAll(pageTotal, i+"");
	        	boolean rs = true;
	        	rs = getPage(url, pages.get(key));
	        	if(!rs) break;
	        }
        }
        System.out.println("本次抓取数据：" + count);
		
		// 最后输出失败的链接到数据库
		GenericArti.insertError2Db(soruce);
	}
	
	public void grapImages() {
      List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), soruce);
		// 抓取图片
		if(null != ids && ids.size() > 0) {
			HandleImages.getContentImgs(ids, baseUrl, imgPath);
//			// 抓取缩略图
//			HandleImages.getThumbImgs(ids, baseUrl, imgPath);
		}
	}
	
	/**
	 * @param url
	 * @param typeName
	 * @return
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName + "====" + url);
		Document doc = GenericGrap.getDocByIp(url);
		Elements els = doc.getElementsByAttributeValue("class","article-lists").get(0).getElementsByTag("li");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				Elements titleEls = el.getElementsByTag("h2");
				if(null == titleEls || titleEls.size() == 0) continue;
				// 获取标题
				Element titleEl = el.getElementsByTag("h2").get(0).getElementsByTag("a").get(0);
				String title = titleEl.text();
				if(titles.contains(title)) {
					return false;
				}
				Elements thumbEls = el.getElementsByTag("img");
				Element thumbEl = null;
				ArticlesTo arti = new ArticlesTo();
				if(thumbEls != null && thumbEls.size() > 0) {
					thumbEl = el.getElementsByTag("img").get(0);
					arti.setThumb(baseUrl + thumbEl.attr("src"));
				}
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(baseUrl + titleEl.attr("href"));
				arti.setTypeName(typeName);
				try {
					Element publishTimeEl = el.getElementsByAttributeValue("class", "pull-right").get(0);
					arti.setPublishTime(publishTimeEl.text());
				} catch (Exception e) {
					e.printStackTrace();
				}
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}

	public void getDetail(ArticlesTo arti) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		String url = arti.getSourceUrl();
//		System.out.println("start get detail data!  " + url);
		try {
			Document doc = GenericGrap.getDocByIp(url);
			String content = getContent(doc, 0);
			// 获取分页信息
			Elements els = doc.getElementsByAttributeValue("class", "pagination");
			if(null != els && els.size()>0) {
				els =els.get(0).getElementsByTag("a");
				// 获取最后一页数据
				Element el = els.get(els.size() - 2);
				String href = el.attr("href");
				String pageTotal = el.text();
				for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
					// 799470_3.html
					String pageUrl = url.substring(0, url.lastIndexOf("/")+1) + href.substring(0, href.lastIndexOf("_")+1) + (i+1) + ".html";
//		        	System.out.println("start get detail data!  " + pageUrl);
					Document pageDoc = GenericGrap.getDocByIp(pageUrl);
					content += "\n"+getContent(pageDoc, i);;
		        }
			}
			content = GenericArti.handleSpace(HtmlUtil.delAllTag(content));
			// 抓取标签
			els = doc.getElementsByAttributeValue("class", "article-tags clearfix").get(0).getElementsByTag("a");
			String tags = null;
			for(Element el : els) {
				if(tags!=null && tags.length() > 0 ) {
					tags += ",";
				} else {
					tags = "";
				}
				tags += el.text();
			}
			arti.setTag(tags);
			arti.setContent("<div class=\"article\">" + content + "</div>");
			GenericArti.insert2Db(arti);
//			GenericGrap.titles.add(arti.getTitle());
			count++;
        } catch (Exception e) {
        	GenericArti.errorUrls.put(url, arti.getTypeName());
        	e.printStackTrace();
        	return;
        }
	}
	
	public static String getContent(Document doc, int page) {
		Element el = doc.getElementsByAttributeValue("class", "article-content").get(0).getElementsByTag("td").get(0);
		// 去掉导读html
		if(page ==0) el.getElementsByTag("p").get(0).remove();
		return el.html();
	}
}
